#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
________________________________________________
 @Time    : 2018/6/6 上午11:36
 @Author  : NickTet
 @Site    : 
 @File    : ProxyRefreshSchedule.py
 @Software: PyCharm
 _______________________________________________
 """
import sys
import time
import logging
from threading import Thread
from apscheduler.schedulers.blocking import BlockingScheduler


from Util.UtilFunction import validUsefulProxy
from Manager.ProxyManager import ProxyManager
from Util.LogHandler import LogHandler

logging.basicConfig()


class ProxyRefreshSchedule(ProxyManager):
    def __init__(self):
        ProxyManager.__init__(self)
        self.log = LogHandler("refresh_schedule")

    def validProxy(self):
        """
        验证raw_proxy_queue中的代理, 将可用的代理放入useful_proxy_queue
        :return: 
        """
        self.db.changeTable(self.raw_proxy_queue)
        raw_proxy_item = self.db.pop()
        self.log.info('ProxyRefreshSchedule: %s start validProxy' % time.ctime())
        # 计算剩余代理，用来减少重复计算
        remaining_proxies = self.getAll()
        while raw_proxy_item:
            raw_proxy = raw_proxy_item.get('proxy')
            if isinstance(raw_proxy, bytes):
                # 兼容Py3
                raw_proxy = raw_proxy.decode('utf8')
            if (raw_proxy not in remaining_proxies) and validUsefulProxy(raw_proxy):
                self.db.changeTable(self.useful_proxy_queue)
                self.db.put(raw_proxy)
                self.log.info('ProxyRefreshSchedule: %s validation pass' % raw_proxy)
            else:
                self.log.info('ProxyRefreshSchedule: %s validation fail' % raw_proxy)
            self.db.changeTable(self.raw_proxy_queue)
            raw_proxy_item = self.db.pop()
            remaining_proxies = self.getAll()
        self.log.info('ProxyRefreshSchedule: %s validProxy complete' % time.ctime())


def refreshPool():
    pp = ProxyRefreshSchedule()
    pp.validProxy()


def main(process_num=30):
    p = ProxyRefreshSchedule()
    p.refresh()
    pl = []
    for num in range(process_num):
        proc = Thread(target=refreshPool(), args=())
        pl.append(proc)
    for num in range(process_num):
        pl[num].daemon = True
        pl[num].start()
    for num in range(process_num):
        pl[num].join()


def run():
    main()
    sch = BlockingScheduler()
    sch.add_job(main, 'interval', minutes=10)
    sch.start()


if __name__ == '__main__':
    run()
